# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
#! --coding:utf-8 --
from urllib2 import urlparse
import sqlite3

class ItemextractPipeline(object):
    def __init__(self):
        pass
    
    def process_item(self, item, spider):
        #print urlparse.urlparse(item['link'])
        self.conn.execute("insert into scan(link) values('%s')" % item['link'])
        #self.conn.commit()
        return item
    def open_spider(self,spider):
        self.conn = sqlite3.connect("webcrawler")
        self.conn.isolation_level = None
        self.conn.execute("create table if not exists scan(id integer primary key autoincrement,link varchar(128))")
        self.conn.execute("delete from scan")
#         print "*************spider open*******************"
        
        #self.conn.commit()
    def close_spider(self,spider):
#         print "*************spider closed******************"
        self.conn.close()
        
    
